{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0,1,2\"\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "from PIL import Image\n",
    "Image.LOAD_TRUNCATED_IMAGES = True\n",
    "import multiprocessing\n",
    "import shutil\n",
    "from glob import glob\n",
    "\n",
    "import numpy as np\n",
    "from keras import backend\n",
    "from keras.callbacks import ReduceLROnPlateau,LearningRateScheduler\n",
    "from keras.callbacks import TensorBoard, Callback, ModelCheckpoint\n",
    "from keras.layers import Dense, GlobalAveragePooling2D,Dropout\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam, RMSprop\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras_radam import RAdam\n",
    "from lookahead import Lookahead\n",
    "import keras.backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from classification_models.keras import Classifiers\n",
    "xception, preprocess_input = Classifiers.get('xception')\n",
    "# import efficientnet.keras as efn\n",
    "# preprocess_input = efn.preprocess_input\n",
    "# from keras_applications.efficientnet import EfficientNetB4,preprocess_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total label: 10651 \n",
      "total img: 10651 \n",
      "total samples: 10651, training samples: 9585, validation samples: 1066\n"
     ]
    }
   ],
   "source": [
    "from data_gen_label_cut  import data_flow\n",
    "batch_size = 32\n",
    "num_classes = 9\n",
    "input_size = 380\n",
    "train_sequence, validation_sequence = data_flow(batch_size,num_classes,input_size,preprocess_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.5/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 380, 380, 3)  0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_1 (Lambda)               (None, 380, 380, 3)  0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 380, 380, 3)  0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "lambda_3 (Lambda)               (None, 380, 380, 3)  0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "model_1 (Model)                 (None, 9)            20879921    lambda_1[0][0]                   \n",
      "                                                                 lambda_2[0][0]                   \n",
      "                                                                 lambda_3[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Concatenate)           (None, 9)            0           model_1[1][0]                    \n",
      "                                                                 model_1[2][0]                    \n",
      "                                                                 model_1[3][0]                    \n",
      "==================================================================================================\n",
      "Total params: 20,879,921\n",
      "Trainable params: 20,825,393\n",
      "Non-trainable params: 54,528\n",
      "__________________________________________________________________________________________________\n",
      "model ok!\n"
     ]
    }
   ],
   "source": [
    "from keras.utils import multi_gpu_model\n",
    "def model_fn(objective, optimizer, metrics):\n",
    "#     base_model = efn.EfficientNetB4(include_top=False,\n",
    "    base_model = xception(include_top=False,\n",
    "                           input_shape=(input_size, input_size, 3),\n",
    "                           classes=num_classes,\n",
    "                           weights='imagenet',)\n",
    "    x = base_model.output\n",
    "    x = GlobalAveragePooling2D( )(x)\n",
    "    predictions = Dense(num_classes, activation='softmax')(x)\n",
    "    model1 = Model(inputs=base_model.input, outputs=predictions)\n",
    "    model2 = multi_gpu_model(model1, gpus=3)\n",
    "    model2.compile(loss=objective, optimizer=optimizer, metrics=metrics)\n",
    "    lookahead = Lookahead(k=5, alpha=0.5) # Initialize Lookahead\n",
    "    lookahead.inject(model2) # add into model\n",
    "    model2.summary( )\n",
    "   #     model.load_weights('./model_snapshots/G0003/weights_036_0.8510.h5')\n",
    "  #     print(\"load ok!!!!!\")\n",
    "    return model1,model2\n",
    "optimizer = RAdam(learning_rate=1e-3)\n",
    "objective = 'categorical_crossentropy'\n",
    "metrics = ['accuracy']\n",
    "model1,model2 = model_fn(objective, optimizer, metrics)\n",
    "print(\"model ok!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyCbk(Callback):\n",
    "    def __init__(self, model):\n",
    "         self.model_to_save = model\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        self.model_to_save.save('./model_snapshots/X0003/model_at_epoch_%d.h5' % epoch)\n",
    "\n",
    "cbk1 = MyCbk(model1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_local = './logs/'+'X0003-xception(380)-cutmix&mixup(1.0)-wp'\n",
    "tensorBoard = TensorBoard(log_dir=log_local)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from warmup_cosine_decay_scheduler import WarmUpCosineDecayScheduler\n",
    "learning_rate = 1e-3\n",
    "sample_count = len(train_sequence) * batch_size\n",
    "epochs = 50\n",
    "warmup_epoch = 8\n",
    "batch_size = batch_size\n",
    "learning_rate_base = learning_rate\n",
    "total_steps = int(epochs * sample_count / batch_size)\n",
    "warmup_steps = int(warmup_epoch * sample_count / batch_size)\n",
    "\n",
    "warm_up_lr = WarmUpCosineDecayScheduler(learning_rate_base=learning_rate_base,\n",
    "                                        total_steps=total_steps,\n",
    "                                        warmup_learning_rate=0,\n",
    "                                        warmup_steps=warmup_steps,\n",
    "                                        hold_base_rate_steps=0,\n",
    "                                        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def scheduler(epoch):\n",
    "#     # 每隔100个epoch，学习率减小为原来的1/10\n",
    "#     if epoch % 4 == 0 and epoch != 0:\n",
    "#         lr = K.get_value(model2.optimizer.lr)\n",
    "#         K.set_value(model2.optimizer.lr, lr * 0.8)\n",
    "#         print(\"lr changed to {}\".format(lr * 0.8))\n",
    "#     return K.get_value(model2.optimizer.lr)\n",
    " \n",
    "# reduce_lr = LearningRateScheduler(scheduler)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/150\n",
      "300/300 [==============================] - 233s 777ms/step - loss: 2.0331 - acc: 0.3132 - val_loss: 1.3807 - val_acc: 0.6426\n",
      "Epoch 2/150\n",
      "300/300 [==============================] - 204s 679ms/step - loss: 1.6366 - acc: 0.5843 - val_loss: 0.9877 - val_acc: 0.8133\n",
      "Epoch 3/150\n",
      "300/300 [==============================] - 208s 693ms/step - loss: 1.5105 - acc: 0.6038 - val_loss: 0.8724 - val_acc: 0.8518\n",
      "Epoch 4/150\n",
      "300/300 [==============================] - 210s 699ms/step - loss: 1.4196 - acc: 0.6746 - val_loss: 0.9198 - val_acc: 0.8255\n",
      "Epoch 5/150\n",
      "300/300 [==============================] - 211s 704ms/step - loss: 1.3771 - acc: 0.6793 - val_loss: 0.9563 - val_acc: 0.8096\n",
      "Epoch 6/150\n",
      "300/300 [==============================] - 207s 691ms/step - loss: 1.3645 - acc: 0.7241 - val_loss: 0.8685 - val_acc: 0.8368\n",
      "Epoch 7/150\n",
      "300/300 [==============================] - 206s 686ms/step - loss: 1.3059 - acc: 0.7502 - val_loss: 0.8863 - val_acc: 0.8433\n",
      "Epoch 8/150\n",
      "300/300 [==============================] - 209s 697ms/step - loss: 1.2544 - acc: 0.7629 - val_loss: 0.8984 - val_acc: 0.8452\n",
      "Epoch 9/150\n",
      "300/300 [==============================] - 205s 684ms/step - loss: 1.2662 - acc: 0.7726 - val_loss: 0.9272 - val_acc: 0.8330\n",
      "Epoch 10/150\n",
      "300/300 [==============================] - 210s 701ms/step - loss: 1.3072 - acc: 0.7492 - val_loss: 0.9252 - val_acc: 0.8227\n",
      "Epoch 11/150\n",
      "300/300 [==============================] - 212s 706ms/step - loss: 1.3078 - acc: 0.7574 - val_loss: 1.0014 - val_acc: 0.8021\n",
      "Epoch 12/150\n",
      "300/300 [==============================] - 208s 694ms/step - loss: 1.2583 - acc: 0.7898 - val_loss: 0.9989 - val_acc: 0.8133\n",
      "Epoch 13/150\n",
      "300/300 [==============================] - 210s 700ms/step - loss: 1.1897 - acc: 0.7871 - val_loss: 0.9254 - val_acc: 0.8349\n",
      "Epoch 14/150\n",
      "300/300 [==============================] - 206s 686ms/step - loss: 1.2462 - acc: 0.7609 - val_loss: 0.8489 - val_acc: 0.8499\n",
      "Epoch 15/150\n",
      "300/300 [==============================] - 217s 722ms/step - loss: 1.1723 - acc: 0.8175 - val_loss: 0.8890 - val_acc: 0.8340\n",
      "Epoch 16/150\n",
      "300/300 [==============================] - 212s 708ms/step - loss: 1.1832 - acc: 0.7918 - val_loss: 0.8759 - val_acc: 0.8424\n",
      "Epoch 17/150\n",
      "300/300 [==============================] - 210s 701ms/step - loss: 1.1837 - acc: 0.8020 - val_loss: 0.9270 - val_acc: 0.8330\n",
      "Epoch 18/150\n",
      "300/300 [==============================] - 205s 684ms/step - loss: 1.2166 - acc: 0.7904 - val_loss: 0.9311 - val_acc: 0.8218\n",
      "Epoch 19/150\n",
      "300/300 [==============================] - 213s 710ms/step - loss: 1.1215 - acc: 0.8404 - val_loss: 0.9057 - val_acc: 0.8386\n",
      "Epoch 20/150\n",
      "300/300 [==============================] - 205s 682ms/step - loss: 1.1038 - acc: 0.8014 - val_loss: 0.9027 - val_acc: 0.8368\n",
      "Epoch 21/150\n",
      "300/300 [==============================] - 208s 692ms/step - loss: 1.1500 - acc: 0.8331 - val_loss: 0.8538 - val_acc: 0.8462\n",
      "Epoch 22/150\n",
      "300/300 [==============================] - 210s 701ms/step - loss: 1.1936 - acc: 0.7384 - val_loss: 0.9290 - val_acc: 0.8330\n",
      "Epoch 23/150\n",
      "300/300 [==============================] - ETA: 0s - loss: 1.1753 - acc: 0.849 - 212s 705ms/step - loss: 1.1749 - acc: 0.8494 - val_loss: 0.9010 - val_acc: 0.8499\n",
      "Epoch 24/150\n",
      "300/300 [==============================] - 215s 716ms/step - loss: 1.1109 - acc: 0.8661 - val_loss: 0.8591 - val_acc: 0.8565\n",
      "Epoch 25/150\n",
      "300/300 [==============================] - 208s 692ms/step - loss: 1.1244 - acc: 0.8366 - val_loss: 0.9667 - val_acc: 0.8255\n",
      "Epoch 26/150\n",
      "300/300 [==============================] - 216s 718ms/step - loss: 1.1111 - acc: 0.8422 - val_loss: 0.8413 - val_acc: 0.8696\n",
      "Epoch 27/150\n",
      "300/300 [==============================] - 212s 708ms/step - loss: 1.0097 - acc: 0.8898 - val_loss: 0.8357 - val_acc: 0.8649\n",
      "Epoch 28/150\n",
      "300/300 [==============================] - 201s 672ms/step - loss: 1.0730 - acc: 0.8541 - val_loss: 0.8748 - val_acc: 0.8659\n",
      "Epoch 29/150\n",
      "300/300 [==============================] - 209s 697ms/step - loss: 1.0692 - acc: 0.8627 - val_loss: 0.9226 - val_acc: 0.8274\n",
      "Epoch 30/150\n",
      "300/300 [==============================] - 209s 697ms/step - loss: 1.1052 - acc: 0.8131 - val_loss: 0.9129 - val_acc: 0.8602\n",
      "Epoch 31/150\n",
      "300/300 [==============================] - 218s 725ms/step - loss: 1.1212 - acc: 0.8453 - val_loss: 0.8498 - val_acc: 0.8715\n",
      "Epoch 32/150\n",
      "300/300 [==============================] - 210s 700ms/step - loss: 1.0462 - acc: 0.8473 - val_loss: 0.8240 - val_acc: 0.8724\n",
      "Epoch 33/150\n",
      "300/300 [==============================] - 208s 695ms/step - loss: 1.0228 - acc: 0.8428 - val_loss: 0.8193 - val_acc: 0.8734\n",
      "Epoch 34/150\n",
      "300/300 [==============================] - 209s 696ms/step - loss: 1.0585 - acc: 0.8237 - val_loss: 0.8541 - val_acc: 0.8705\n",
      "Epoch 35/150\n",
      "300/300 [==============================] - 205s 683ms/step - loss: 1.0429 - acc: 0.8825 - val_loss: 0.8288 - val_acc: 0.8724\n",
      "Epoch 36/150\n",
      "300/300 [==============================] - 210s 700ms/step - loss: 0.9625 - acc: 0.9029 - val_loss: 0.8023 - val_acc: 0.8856\n",
      "Epoch 37/150\n",
      "300/300 [==============================] - 202s 675ms/step - loss: 1.0312 - acc: 0.7979 - val_loss: 0.8066 - val_acc: 0.8837\n",
      "Epoch 38/150\n",
      "300/300 [==============================] - 207s 691ms/step - loss: 0.9753 - acc: 0.8929 - val_loss: 0.8081 - val_acc: 0.8752\n",
      "Epoch 39/150\n",
      "300/300 [==============================] - 204s 680ms/step - loss: 0.9930 - acc: 0.8557 - val_loss: 0.8120 - val_acc: 0.8752\n",
      "Epoch 40/150\n",
      "300/300 [==============================] - 214s 712ms/step - loss: 1.0722 - acc: 0.8304 - val_loss: 0.8282 - val_acc: 0.8696\n",
      "Epoch 41/150\n",
      "300/300 [==============================] - 206s 687ms/step - loss: 1.0410 - acc: 0.8681 - val_loss: 0.9017 - val_acc: 0.8837\n",
      "Epoch 42/150\n",
      "300/300 [==============================] - 209s 696ms/step - loss: 1.0290 - acc: 0.8446 - val_loss: 0.9287 - val_acc: 0.8734\n",
      "Epoch 43/150\n",
      "300/300 [==============================] - 212s 708ms/step - loss: 0.9955 - acc: 0.8784 - val_loss: 0.8265 - val_acc: 0.8705\n",
      "Epoch 44/150\n",
      "300/300 [==============================] - 209s 697ms/step - loss: 1.0214 - acc: 0.7894 - val_loss: 0.8143 - val_acc: 0.8799\n",
      "Epoch 45/150\n",
      "300/300 [==============================] - 220s 732ms/step - loss: 0.9800 - acc: 0.8766 - val_loss: 0.8051 - val_acc: 0.8790\n",
      "Epoch 46/150\n",
      "300/300 [==============================] - 210s 700ms/step - loss: 1.0098 - acc: 0.8777 - val_loss: 0.8394 - val_acc: 0.8799\n",
      "Epoch 47/150\n",
      "300/300 [==============================] - 208s 693ms/step - loss: 0.9782 - acc: 0.8645 - val_loss: 0.8072 - val_acc: 0.8837\n",
      "Epoch 48/150\n",
      "300/300 [==============================] - 214s 713ms/step - loss: 0.9350 - acc: 0.9073 - val_loss: 0.8003 - val_acc: 0.8865\n",
      "Epoch 49/150\n",
      "300/300 [==============================] - 212s 706ms/step - loss: 1.0529 - acc: 0.8374 - val_loss: 0.8125 - val_acc: 0.8818\n",
      "Epoch 50/150\n",
      "300/300 [==============================] - 205s 685ms/step - loss: 0.9940 - acc: 0.8627 - val_loss: 0.8113 - val_acc: 0.8837\n",
      "Epoch 51/150\n",
      "300/300 [==============================] - 207s 689ms/step - loss: 1.0182 - acc: 0.7853 - val_loss: 0.8317 - val_acc: 0.8856\n",
      "Epoch 52/150\n",
      "300/300 [==============================] - 205s 683ms/step - loss: 1.0090 - acc: 0.8580 - val_loss: 0.8462 - val_acc: 0.8827\n",
      "Epoch 53/150\n",
      "300/300 [==============================] - 204s 679ms/step - loss: 1.0410 - acc: 0.7721 - val_loss: 0.8192 - val_acc: 0.8837\n",
      "Epoch 54/150\n",
      "300/300 [==============================] - 212s 708ms/step - loss: 1.0081 - acc: 0.8353 - val_loss: 0.7888 - val_acc: 0.8837\n",
      "Epoch 55/150\n",
      "300/300 [==============================] - 210s 701ms/step - loss: 1.0366 - acc: 0.8415 - val_loss: 0.8184 - val_acc: 0.8837\n",
      "Epoch 56/150\n",
      "300/300 [==============================] - 206s 685ms/step - loss: 0.9463 - acc: 0.8770 - val_loss: 0.8119 - val_acc: 0.8809\n",
      "Epoch 57/150\n",
      "300/300 [==============================] - 209s 696ms/step - loss: 0.9736 - acc: 0.8420 - val_loss: 0.8083 - val_acc: 0.8809\n",
      "Epoch 58/150\n",
      "300/300 [==============================] - 208s 693ms/step - loss: 0.9873 - acc: 0.8601 - val_loss: 0.8315 - val_acc: 0.8780\n",
      "Epoch 59/150\n",
      "300/300 [==============================] - 205s 683ms/step - loss: 1.0364 - acc: 0.8089 - val_loss: 0.8173 - val_acc: 0.8771\n",
      "Epoch 60/150\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "300/300 [==============================] - 204s 680ms/step - loss: 1.0307 - acc: 0.7846 - val_loss: 0.8380 - val_acc: 0.8790\n",
      "Epoch 61/150\n",
      "300/300 [==============================] - 209s 696ms/step - loss: 1.0264 - acc: 0.7757 - val_loss: 0.8261 - val_acc: 0.8790\n",
      "Epoch 62/150\n",
      "300/300 [==============================] - 209s 696ms/step - loss: 0.9889 - acc: 0.8360 - val_loss: 0.8177 - val_acc: 0.8734\n",
      "Epoch 63/150\n",
      "300/300 [==============================] - 211s 703ms/step - loss: 1.0211 - acc: 0.8804 - val_loss: 0.8256 - val_acc: 0.8705\n",
      "Epoch 64/150\n",
      "300/300 [==============================] - 206s 686ms/step - loss: 0.9711 - acc: 0.8306 - val_loss: 0.7929 - val_acc: 0.8818\n",
      "Epoch 65/150\n",
      "300/300 [==============================] - 206s 687ms/step - loss: 0.9779 - acc: 0.8832 - val_loss: 0.8000 - val_acc: 0.8846\n",
      "Epoch 66/150\n",
      "300/300 [==============================] - 202s 673ms/step - loss: 1.0143 - acc: 0.8322 - val_loss: 0.8548 - val_acc: 0.8790\n",
      "Epoch 67/150\n",
      "300/300 [==============================] - 205s 682ms/step - loss: 0.9750 - acc: 0.9030 - val_loss: 0.7977 - val_acc: 0.8827\n",
      "Epoch 68/150\n",
      "300/300 [==============================] - 201s 672ms/step - loss: 0.9255 - acc: 0.8752 - val_loss: 0.8009 - val_acc: 0.8856\n",
      "Epoch 69/150\n",
      "300/300 [==============================] - 206s 685ms/step - loss: 1.0090 - acc: 0.8477 - val_loss: 0.8396 - val_acc: 0.8865\n",
      "Epoch 70/150\n",
      "300/300 [==============================] - 207s 691ms/step - loss: 0.9594 - acc: 0.8594 - val_loss: 0.7932 - val_acc: 0.8809\n",
      "Epoch 71/150\n",
      "300/300 [==============================] - 207s 691ms/step - loss: 0.9897 - acc: 0.8805 - val_loss: 0.8061 - val_acc: 0.8799\n",
      "Epoch 72/150\n",
      "300/300 [==============================] - 207s 689ms/step - loss: 1.0260 - acc: 0.8499 - val_loss: 0.8513 - val_acc: 0.8771\n",
      "Epoch 73/150\n",
      "300/300 [==============================] - 210s 698ms/step - loss: 0.9979 - acc: 0.8037 - val_loss: 0.8708 - val_acc: 0.8752\n",
      "Epoch 74/150\n",
      "300/300 [==============================] - 201s 671ms/step - loss: 1.0088 - acc: 0.7537 - val_loss: 0.8178 - val_acc: 0.8818\n",
      "Epoch 75/150\n",
      "300/300 [==============================] - 202s 673ms/step - loss: 0.9669 - acc: 0.8157 - val_loss: 0.7959 - val_acc: 0.8799\n",
      "Epoch 76/150\n",
      "300/300 [==============================] - 208s 695ms/step - loss: 0.9901 - acc: 0.9155 - val_loss: 0.8079 - val_acc: 0.8837\n",
      "Epoch 77/150\n",
      "300/300 [==============================] - 206s 686ms/step - loss: 0.9893 - acc: 0.8237 - val_loss: 0.7908 - val_acc: 0.8865\n",
      "Epoch 78/150\n",
      "300/300 [==============================] - 204s 679ms/step - loss: 0.9147 - acc: 0.9371 - val_loss: 0.7920 - val_acc: 0.8856\n",
      "Epoch 79/150\n",
      "300/300 [==============================] - 210s 701ms/step - loss: 0.9913 - acc: 0.9100 - val_loss: 0.7930 - val_acc: 0.8818\n",
      "Epoch 80/150\n",
      "300/300 [==============================] - 203s 676ms/step - loss: 0.9481 - acc: 0.8963 - val_loss: 0.8033 - val_acc: 0.8799\n",
      "Epoch 81/150\n",
      "300/300 [==============================] - 202s 672ms/step - loss: 0.9365 - acc: 0.8924 - val_loss: 0.7873 - val_acc: 0.8827\n",
      "Epoch 82/150\n",
      "300/300 [==============================] - 207s 691ms/step - loss: 0.9806 - acc: 0.8537 - val_loss: 0.8241 - val_acc: 0.8856\n",
      "Epoch 83/150\n",
      "300/300 [==============================] - 210s 699ms/step - loss: 0.9898 - acc: 0.8612 - val_loss: 0.7941 - val_acc: 0.8874\n",
      "Epoch 84/150\n",
      "300/300 [==============================] - 210s 700ms/step - loss: 0.9570 - acc: 0.9151 - val_loss: 0.7964 - val_acc: 0.8752\n",
      "Epoch 85/150\n",
      "300/300 [==============================] - 208s 693ms/step - loss: 0.9587 - acc: 0.8269 - val_loss: 0.8001 - val_acc: 0.8780\n",
      "Epoch 86/150\n",
      "300/300 [==============================] - 209s 698ms/step - loss: 0.9849 - acc: 0.8593 - val_loss: 0.7996 - val_acc: 0.8809\n",
      "Epoch 87/150\n",
      "300/300 [==============================] - 212s 708ms/step - loss: 0.9745 - acc: 0.8394 - val_loss: 0.7989 - val_acc: 0.8762\n",
      "Epoch 88/150\n",
      "300/300 [==============================] - 206s 687ms/step - loss: 1.0140 - acc: 0.8415 - val_loss: 0.8162 - val_acc: 0.8715\n",
      "Epoch 89/150\n",
      "300/300 [==============================] - 209s 696ms/step - loss: 0.9798 - acc: 0.8195 - val_loss: 0.8153 - val_acc: 0.8743\n",
      "Epoch 90/150\n",
      "300/300 [==============================] - 207s 689ms/step - loss: 0.9536 - acc: 0.8453 - val_loss: 0.8054 - val_acc: 0.8799\n",
      "Epoch 91/150\n",
      "300/300 [==============================] - 202s 675ms/step - loss: 0.9676 - acc: 0.9111 - val_loss: 0.8073 - val_acc: 0.8734\n",
      "Epoch 92/150\n",
      "300/300 [==============================] - 210s 699ms/step - loss: 0.9678 - acc: 0.9352 - val_loss: 0.8093 - val_acc: 0.8780\n",
      "Epoch 93/150\n",
      "300/300 [==============================] - 203s 678ms/step - loss: 0.9921 - acc: 0.8412 - val_loss: 0.8118 - val_acc: 0.8780\n",
      "Epoch 94/150\n",
      "300/300 [==============================] - 202s 674ms/step - loss: 0.9622 - acc: 0.9280 - val_loss: 0.8031 - val_acc: 0.8780\n",
      "Epoch 95/150\n",
      "300/300 [==============================] - 207s 689ms/step - loss: 0.9636 - acc: 0.8269 - val_loss: 0.7987 - val_acc: 0.8809\n",
      "Epoch 96/150\n",
      "300/300 [==============================] - 207s 689ms/step - loss: 0.9536 - acc: 0.8721 - val_loss: 0.8018 - val_acc: 0.8818\n",
      "Epoch 97/150\n",
      "300/300 [==============================] - 210s 701ms/step - loss: 0.9902 - acc: 0.7992 - val_loss: 0.8074 - val_acc: 0.8809\n",
      "Epoch 98/150\n",
      "300/300 [==============================] - 211s 704ms/step - loss: 0.9745 - acc: 0.8550 - val_loss: 0.8039 - val_acc: 0.8799\n",
      "Epoch 99/150\n",
      "300/300 [==============================] - 208s 693ms/step - loss: 1.0047 - acc: 0.8724 - val_loss: 0.7962 - val_acc: 0.8780\n",
      "Epoch 100/150\n",
      "300/300 [==============================] - 210s 700ms/step - loss: 0.9688 - acc: 0.8949 - val_loss: 0.8023 - val_acc: 0.8752\n",
      "Epoch 101/150\n",
      "300/300 [==============================] - 208s 694ms/step - loss: 0.9861 - acc: 0.8489 - val_loss: 0.7997 - val_acc: 0.8846\n",
      "Epoch 102/150\n",
      "300/300 [==============================] - 201s 669ms/step - loss: 0.9595 - acc: 0.8605 - val_loss: 0.8091 - val_acc: 0.8780\n",
      "Epoch 103/150\n",
      "119/300 [==========>...................] - ETA: 1:51 - loss: 0.8350 - acc: 0.8180"
     ]
    }
   ],
   "source": [
    "model2.fit_generator(\n",
    "    train_sequence,\n",
    "    steps_per_epoch=len(train_sequence),\n",
    "    epochs=150,\n",
    "    verbose=1,\n",
    "    callbacks=[cbk1, tensorBoard, warm_up_lr],\n",
    "    validation_data=validation_sequence,\n",
    "    max_queue_size=10,\n",
    "    workers=int(multiprocessing.cpu_count( ) * 0.9),\n",
    "    use_multiprocessing=True,\n",
    "    shuffle=True\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1.0,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1.0,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
